# Copyright (c) 2018 Presto Labs Pte. Ltd.
# Author: jhkim
import pydevd
import collections
import functools
import os
import datetime
import logging
import sys
import numpy
import pandas

from absl import app, flags

from coin.base.book.types import TradeSide
from coin.exchange.okex.kr_rest.product import OkexProduct
from coin.exchange.huobi.kr_rest.product import HuobiProduct
from coin.exchange.binance.kr_rest.product import BinanceProduct
from coin.exchange.bitfinex_v2.kr_rest.product import BitfinexProduct
from coin.exchange.shared.kr_rest.product import SharedProduct
from coin.exchange.okex_futures.kr_rest.futures_product import OkexFuturesProduct
from coin.exchange.bitmex.kr_rest.futures_product import BitmexFuturesProduct
from coin.exchange.bitflyer_v1.kr_rest.futures_product import BitflyerFuturesProduct
from coin.support.proto_log.logic.util import run_from_og_log_archive
from coin.strategy.mm.dumper_base import run_from_archive
from coin.proto.coin_order_gateway_pb2 import OrderGatewayLog, OrderEvent
from coin.proto.coin_order_enums_pb2 import BUY_ORDER, SELL_ORDER, UNKNOWN_ORDER_SIDE
from coin.strategy.mm.tool.duration_tracker import DurationTracker
from coin.strategy.mm.tool.util import get_order_sign

import coin.strategy.mm.feed as feed

import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt

flags.DEFINE_bool('detail', False, 'show detail stats')

FLAGS = flags.FLAGS


def get_weighted_mean_robust(values, weights):
  if weights.sum() > 0:
    return numpy.average(values, weights=weights)
  else:
    return float('nan')


def calculate_period_pnl(filldf):
  filldf = filldf.sort_values('timestamp')
  pnl = list(filldf['fill_qty'] * filldf['next_fill_ret'])
  timestamp = pandas.to_datetime(filldf['timestamp'])
  pnl = pandas.Series(pnl, index=timestamp)
  agg_pnl = pnl.resample('20T').sum()
  return agg_pnl


def get_stat_oe(oedf, need_plot=True):
  assert len(oedf['symbol'].unique()) == 1
  filldf = oedf.loc[oedf['event_type'] == OrderEvent.ORDER_FILLED].reset_index(drop=True)

  next_buy_price = None
  next_sell_price = None
  next_midp = None
  rows = []

  if len(filldf) > 0:
    last_fill_price = filldf.iloc[-1]['fill_price']
    pos_pnl_ret_bp = (filldf.iloc[-1]['fill_price']
                      - filldf.iloc[0]['fill_price']) / filldf.iloc[0]['fill_price'] * 1e4
    prc_diff = (filldf.iloc[-1]['fill_price'] - filldf.iloc[0]['fill_price'])
  else:
    last_fill_price = None
    pos_pnl_ret_bp = None
    prc_diff = None

  next_fills = []

  for _, row in filldf.iloc[::-1].iterrows():
    if next_sell_price is not None and next_buy_price is not None:
      next_midp = (next_sell_price + next_buy_price) * 0.5

    # next_mtmp = next_midp
    if len(next_fills) == 0:
      next_mtmp = None
    else:
      next_mtmp = numpy.mean(next_fills[-(min(len(next_fills), 60)):])

    if row['sign'] == 1:
      ret = 0 if next_mtmp is None else next_mtmp - row['fill_price']
      last_fill_ret = last_fill_price - row['fill_price']
      next_buy_price = row['fill_price']
    elif row['sign'] == -1:
      ret = 0 if next_mtmp is None else row['fill_price'] - next_mtmp
      next_sell_price = row['fill_price']
      last_fill_ret = row['fill_price'] - last_fill_price
    else:
      ret = 0
      last_fill_ret = 0
    row['next_fill_ret'] = ret
    row['last_fill_ret'] = last_fill_ret
    next_fills.append(row['fill_price'])
    rows.append(row)
  filldf = pandas.DataFrame(rows[::-1])

  subdf = oedf.loc[oedf['event_type'] == OrderEvent.ORDER_SUBMITTED].reset_index(drop=True)
  candf = oedf.loc[oedf['event_type'] == OrderEvent.CANCEL_CONFIRMED].reset_index(drop=True)

  pnl = calculate_period_pnl(filldf)
  pnl.to_csv('pnl.csv')

  statcol = {
      'symbol': oedf['symbol'].unique()[0],
      'pnl_avg_pos': oedf['last_pos'].mean() * (prc_diff or 0),
      'lot_size_avg': subdf['qty'].mean(),
      'cnt_fill': filldf.shape[0],
      'cnt_order_sub': subdf.shape[0],
      'avg_pos': oedf['last_pos'].mean(),
  }

  if FLAGS.detail:
    statcol.update({
        'pnl_avg_pos_ret_bp': pos_pnl_ret_bp,
        'cancel_cnt': candf.shape[0],
        'cancel_qty': candf['qty'].sum(),
        'order_qty': subdf['qty'].sum(),
    })

  if len(filldf) > 0:
    if (len(numpy.unique(filldf['fill_qty'])) > 1
        and len(numpy.unique(filldf['next_fill_ret'])) > 1):
      fill_qty_ev_corr = numpy.corrcoef(filldf['fill_qty'], filldf['next_fill_ret'])[0, 1]
    else:
      fill_qty_ev_corr = numpy.nan
    is_full_fill = filldf['fill_qty'] == filldf['fill_qty'].max()
    full_fill_ratio = is_full_fill.mean()
    fill_ev_bp = filldf['next_fill_ret'] / filldf['fill_price'] * 1e4
    fill_ev_actual_bp = filldf['last_fill_ret'] / filldf['fill_price'] * 1e4

    agg_idx = ~numpy.isclose(filldf['fill_price'], filldf['price'], atol=0)

    if need_plot:
      matplotlib.rcParams.update({'font.size': 7})
      plt.subplot(311)
      plt.plot(pandas.DatetimeIndex(filldf['timestamp']), filldf['fill_price'], lw=0.5)
      plt.title('fill price, %s' % oedf['symbol'][0])
      plt.xticks(rotation=10)
      plt.subplot(312)
      plt.plot(pandas.DatetimeIndex(filldf['timestamp']),
               (filldf['fill_qty'] * filldf['next_fill_ret']).cumsum(),
               lw=1)
      plt.xticks(rotation=10)
      plt.twinx()
      plt.plot(pandas.DatetimeIndex(oedf['timestamp']), oedf['last_pos'], color='red', lw=0.5)
      plt.title('estimated pnl, position(red)')
      plt.subplot(325)
      plt.plot((filldf['fill_qty'] * filldf['fill_price']).cumsum())
      plt.title('cum turnover')
      plt.subplot(326)
      canqty = oedf['qty'] * (oedf['event_type'] == OrderEvent.CANCEL_CONFIRMED)
      plt.plot(pandas.DatetimeIndex(oedf['timestamp']), oedf['fill_qty'].cumsum(), 'r')
      plt.xticks(rotation=20)
      plt.twinx()
      plt.plot(pandas.DatetimeIndex(oedf['timestamp']), canqty.cumsum(), 'b')
      plt.title('fillqty/canqty sum')
      os.makedirs("oe_stat", exist_ok=True)
      plt.savefig("oe_stat/%s_%s.png" % (oedf['symbol'][0], FLAGS.trading_date))
      plt.close()

      plt.subplot(211)
      plt.plot(pandas.DatetimeIndex(filldf['timestamp']),
               filldf['fill_price'],
               'r',
               pandas.DatetimeIndex(filldf['timestamp']),
               filldf['price'],
               'b')
      plt.subplot(212)
      plt.plot(pandas.DatetimeIndex(filldf['timestamp'][agg_idx]),
               filldf['fill_price'][agg_idx],
               'b.')
      plt.twinx()
      plt.plot(pandas.DatetimeIndex(filldf['timestamp']), (filldf['fill_qty'] *
                                                           (filldf['fill_price'] > 0)).cumsum(),
               'r')
      plt.plot(pandas.DatetimeIndex(filldf['timestamp']), (filldf['fill_qty'] * (agg_idx)).cumsum(),
               'g')
      plt.twinx()
      plt.plot(pandas.DatetimeIndex(filldf['timestamp']),
               (fill_ev_bp * filldf['fill_qty']).cumsum(),
               'y')
      plt.savefig("oe_stat/agg_view_%s_%s.png" % (oedf['symbol'][0], FLAGS.trading_date))
      plt.close()

    fill_stats = {
        'agg_ratio_cnt':
            agg_idx.mean(),
        'agg_ratio_qty':
            0 if filldf['fill_qty'].sum() == 0 else filldf['fill_qty'][agg_idx].sum()
            / filldf['fill_qty'].sum(),
        'pnl_fill_ev_bp':
            get_weighted_mean_robust(fill_ev_bp, weights=filldf['fill_qty']),
        'pnl_fill_est': (filldf['next_fill_ret'] * filldf['fill_qty']).sum(),
        'full_fill_ratio':
            full_fill_ratio,
        'fill_qty':
            filldf['fill_qty'].sum(),
        'fill_qty_avg':
            filldf['fill_qty'].mean(),
        'fill_pq': (filldf['fill_qty'] * filldf['fill_price']).sum(),
        'fill_qty_ratio':
            filldf['fill_qty'].sum() / subdf['qty'].sum(),
    }

    if FLAGS.detail:
      fill_stats.update({
          'fill_ev_actual_bp':
              get_weighted_mean_robust(fill_ev_actual_bp, weights=filldf['fill_qty']),
          'fill_qty_ev_corr':
              fill_qty_ev_corr,
          'fill_ev_bp_buy':
              get_weighted_mean_robust(fill_ev_bp[filldf['sign'] == 1],
                                       weights=filldf['fill_qty'][filldf['sign'] == 1]),
          'fill_ev_bp_sell':
              get_weighted_mean_robust(fill_ev_bp[filldf['sign'] == -1],
                                       weights=filldf['fill_qty'][filldf['sign'] == -1]),
          'fill_ev_partial_bp':
              get_weighted_mean_robust(fill_ev_bp[~is_full_fill],
                                       weights=filldf['fill_qty'][~is_full_fill]),
          'fill_ev_full_bp':
              get_weighted_mean_robust(fill_ev_bp[is_full_fill],
                                       weights=filldf['fill_qty'][is_full_fill]),
          'fill_qty_buy':
              filldf['fill_qty'][filldf['sign'] == 1].sum(),
          'fill_qty_sell':
              filldf['fill_qty'][filldf['sign'] == -1].sum(),
          'fill_qty_avg/lot_size_avg':
              filldf['fill_qty'].mean() / subdf['qty'].mean(),
      })
    statcol.update(fill_stats)

  return statcol


class OrderIdConverter(object):
  def __init__(self):
    self._internal_id_map = collections.defaultdict(dict)
    self._last_time = collections.defaultdict(int)

  def get_order_id(self, event):
    if self._last_time[event.symbol] > event.event_time:
      return None
    self._last_time[event.symbol] = event.event_time
    return self.get_order_id_impl(event, self._internal_id_map[event.symbol])

  def get_order_id_impl(self, event, internal_id_map):
    if event.type == OrderEvent.ORDER_SUBMITTED:
      if len(internal_id_map) == 0:
        order_id = 0
      else:
        order_id = max(internal_id_map.values()) + 1
      internal_id_map[event.internal_order_id] = order_id
    if event.internal_order_id not in internal_id_map:
      logging.debug("invalid internal order id: %s", event.internal_order_id)
      return None
    return internal_id_map[event.internal_order_id]

  def get_order_id_from_json(self, event_json):
    symbol = event_json['symbol']
    event_time = int(event_json['event_time'])
    if self._last_time[symbol] > event_time:
      return None
    self._last_time[symbol] = event_time
    return self.get_order_id_impl_from_json(event_json, self._internal_id_map[symbol])

  def get_order_id_impl_from_json(self, event_json, internal_id_map):
    internal_order_id = event_json['internal_order_id']
    if event_json['type'] == 'ORDER_SUBMITTED':
      if len(internal_id_map) == 0:
        order_id = 0
      else:
        order_id = max(internal_id_map.values()) + 1
      internal_id_map[internal_order_id] = order_id
    if internal_order_id not in internal_id_map:
      logging.debug("invalid internal order id: %s", internal_order_id)
      return None
    return internal_id_map[internal_order_id]


class PtaStrategy(object):
  def __init__(self, products, market_type, exchange):
    self._market_type = market_type
    self._exchange = exchange
    self._header = [
        "timestamp",
        "symbol",
        "event_type",
        "price",
        "qty",
        "side",
        "sign",
        "order_id",
        "fill_price",
        "fill_qty",
        "last_pos"
    ]
    self._rows = collections.defaultdict(list)
    self._feed_rows = collections.defaultdict(list)
    self._symbol_count = collections.defaultdict(int)
    self._type_dict = {
        key: value for value,
        key in zip(OrderEvent.OrderEventType.keys(), OrderEvent.OrderEventType.values())
    }
    self._order_id_converter = OrderIdConverter()
    self._stat_oe = {}
    self._product_map = {}
    self._symbol_map = {}
    self._products = products
    self._symbol_mtm = {}
    self._stat_df = {}
    self._balance_map = {}
    self._currency_map = {}
    self._arraydict = collections.defaultdict(dict)
    self._first_log_timestamp = None
    self._last_log_timestamp = None
    self._duration_tracker = DurationTracker()
    self._balances = []
    self._prev_mid_price = None
    self._prev_mid_price_timestamp = None
    self._volatility = {}

  def on_book_reset(self, book_builder_name, book_builder):
    for product in self._products:
      subscription_key = feed.get_subscription_key(product)
      if book_builder_name == subscription_key:
        book_builder.subscribe(product, functools.partial(self.on_book, product))
        """
        if hasattr(book_builder, "subscribe_trade"):
          book_builder.subscribe_trade(
              product, functools.partial(self.on_trade, product))
        """

  def try_assign_value(self, product, timestamp, col, val):
    symbol = self._product_map[product]
    if symbol not in self._stat_df:
      return
    sdf = self._stat_df[symbol]
    if sdf['timestamp'][0] > timestamp:
      return
    ind = sdf['timestamp'].searchsorted(timestamp, side='left')[0]
    if ind >= len(sdf):
      return
    if col not in self._arraydict[symbol]:
      self._arraydict[symbol][col] = numpy.array(sdf["timestamp"] * numpy.nan)
    self._arraydict[symbol][col][ind] = val

  def calculate_period_volatility(self):
    feed_header = ['timestamp', 'rtn']
    for symbol in self._feed_rows.keys():
      feeddf = pandas.DataFrame(self._feed_rows[symbol], columns=feed_header)
      feeddf = feeddf.sort_values('timestamp')
      rtn = list(feeddf['rtn'])
      timestamp = pandas.to_datetime(feeddf['timestamp'])
      rtn = pandas.Series(rtn, index=timestamp)
      agg_rtn = rtn.resample('20T').std()
      agg_rtn.to_csv('%s_volatility.csv' % symbol)

  def on_book(self, product, book):
    symbol = product.symbol
    exch = product.exchange
    if book.ask0().price is None or book.bid0().price is None:
      return
    if self._prev_mid_price is None or self._prev_mid_price_timestamp is None:
      mid_price = (book.ask0().price + book.bid0().price) / 2
      self._prev_mid_price = mid_price
      self._prev_mid_price_timestamp = book.timestamp
    else:
      if book.timestamp - self._prev_mid_price_timestamp > 10**10:
        mid_price = (book.ask0().price + book.bid0().price) / 2
        rtn = (mid_price / self._prev_mid_price) - 1
        csv_line = [book.timestamp, rtn]
        self._feed_rows[symbol].append(csv_line)
        self._prev_mid_price = mid_price
        self._prev_mid_price_timestamp = book.timestamp
    """
    self.try_assign_value(
        product, book.timestamp, "%s_ask0p" % exch, book.ask0().price)
    self.try_assign_value(
        product, book.timestamp, "%s_bid0p" % exch, book.bid0().price)
    """

  def on_trade(self, product, trade):
    symbol = self._product_map[product]
    if symbol not in self._stat_df:
      return
    exch = product.exchange
    if trade.side == TradeSide.TRADE_BUY_SIDE:
      self.try_assign_value(product, trade.timestamp, "%s_buyp" % exch, trade.price)
    elif trade.side == TradeSide.TRADE_SELL_SIDE:
      self.try_assign_value(product, trade.timestamp, "%s_sellp" % exch, trade.price)
    else:
      raise ValueError(trade.side)

  def on_log(self, timestamp, log):
    pb = OrderGatewayLog()
    try:
      pb.ParseFromString(log)
    except Exception:
      print(log)
      return
    if pb.type in [OrderGatewayLog.BALANCE, OrderGatewayLog.POSITION]:
      if self._market_type in ['Spot', 'Futures']:
        balance = pb.balance
        for each_balance in balance.each_balance:
          self._balance_map[each_balance.currency] = each_balance.total
        if self._exchange == 'Bitflyer':
          self._balances.append((timestamp, self._balance_map['JPY']))
      elif self._market_type == 'Margin':
        position = pb.position
        for each_position in position.each_position:
          if each_position.symbol in self._currency_map:
            currency = self._currency_map[each_position.symbol]
            self._balance_map[currency] = each_position.net_position
    elif pb.type == OrderGatewayLog.ORDER_EVENT:
      event = pb.event
      self._duration_tracker.on_log(timestamp, event)
      if self._exchange == 'Bitflyer':
        symbol = event.symbol
        timestamp = event.event_time
        price = event.order_price
        qty = event.order_qty
        side = event.order_side
        if event.internal_order_id == '':
          order_id = None
        else:
          order_id = self._order_id_converter.get_order_id(event)
      else:
        if event.internal_order_id == '':
          return
        symbol = event.symbol
        timestamp = event.event_time
        price = event.order_price
        qty = event.order_qty
        side = event.order_side
        order_id = self._order_id_converter.get_order_id(event)
        if order_id is None:
          return
      event_type = event.type
      fill_price = event.fill_price
      fill_qty = event.fill_qty
      sign = get_order_sign(event.order_side)
      if price == 0:
        price = float('nan')
      if fill_price == 0:
        fill_price = float('nan')
      if symbol not in self._currency_map:
        if self._market_type in ['Spot', 'Margin']:
          self._currency_map[symbol] = SharedProduct.FromStr(symbol).base.currency
        elif self._market_type == 'Futures':
          if self._exchange == 'Okex':
            self._currency_map[symbol] = OkexFuturesProduct.FromStr(symbol).base.currency
          elif self._exchange == 'Bitmex':
            self._currency_map[symbol] = BitmexFuturesProduct.FromStr(symbol).base.currency
          elif self._exchange == 'Bitflyer':
            self._currency_map[symbol] = BitflyerFuturesProduct.FromStr(symbol).base.currency
          else:
            raise ValueError(self._exchange)
        else:
          raise ValueError(self._market_type)
      base_currency = self._currency_map[symbol]
      last_pos = self._balance_map.get(base_currency, None)
      csv_line = [
          timestamp,
          symbol,
          event_type,
          price,
          qty,
          side,
          sign,
          order_id,
          fill_price,
          fill_qty,
          last_pos
      ]
      self._symbol_count[symbol] += 1
      self._rows[symbol].append(csv_line)
      if self._first_log_timestamp is None:
        self._first_log_timestamp = timestamp
      self._last_log_timestamp = timestamp

  def on_log_json(self, timestamp, log_json):
    if log_json['type'] in ['BALANCE', 'POSITION']:
      if self._market_type in ['Spot', 'Futures']:
        balance = log_json['balance']
        for each_balance in balance['each_balance']:
          self._balance_map[each_balance['currency']] = each_balance['total']
      elif self._market_type == 'Margin':
        position = log_json['position']
        for each_position in position['each_position']:
          currency = self._currency_map[each_position['symbol']]
          self._balance_map[currency] = each_position['net_position']
    elif log_json['type'] == 'ORDER_EVENT':
      event = log_json['event']
      self._duration_tracker.on_log_json(timestamp, event)
      if 'internal_order_id' not in event:
        return
      symbol = event['symbol']
      timestamp = int(event['event_time'])
      price = event['order_price']
      qty = event['order_qty']
      order_id = self._order_id_converter.get_order_id_from_json(event)
      if order_id is None:
        return

      side = event['order_side']
      if side == 'BUY_ORDER':
        sign = 1
        side = BUY_ORDER
      elif side == 'SELL_ORDER':
        sign = -1
        side = SELL_ORDER
      elif side == 'UNKNOWN_ORDER_SIDE':
        sign = 0
        side = UNKNOWN_ORDER_SIDE
      else:
        raise ValueError(event['order_side'], event)

      event_type = event['type']
      if event_type == 'ORDER_FILLED':
        event_type = OrderEvent.ORDER_FILLED
      elif event_type == 'ORDER_SUBMITTED':
        event_type = OrderEvent.ORDER_SUBMITTED
      elif event_type == 'CANCEL_CONFIRMED':
        event_type = OrderEvent.CANCEL_CONFIRMED
      elif event_type == 'ORDER_SUBMITTED':
        event_type = OrderEvent.ORDER_SUBMITTED

      fill_price = event['fill_price'] if 'fill_price' in event else 0
      fill_qty = event['fill_qty'] if 'fill_qty' in event else 0
      if price == 0:
        price = float('nan')
      if fill_price == 0:
        fill_price = float('nan')
      if symbol not in self._currency_map:
        if self._market_type in ['Spot', 'Margin']:
          self._currency_map[symbol] = SharedProduct.FromStr(symbol).base.currency
        elif self._market_type == 'Futures':
          if self._exchange == 'Okex':
            self._currency_map[symbol] = OkexFuturesProduct.FromStr(symbol).base.currency
          elif self._exchange == 'Bitmex':
            self._currency_map[symbol] = BitmexFuturesProduct.FromStr(symbol).base.currency
          elif self._exchange == 'Bitflyer':
            self._currency_map[symbol] = BitflyerFuturesProduct.FromStr(symbol).base.currency
          else:
            raise ValueError(self._exchange)
        else:
          raise ValueError(self._market_type)
      base_currency = self._currency_map[symbol]
      last_pos = self._balance_map.get(base_currency, None)
      csv_line = [
          timestamp,
          symbol,
          event_type,
          price,
          qty,
          side,
          sign,
          order_id,
          fill_price,
          fill_qty,
          last_pos
      ]
      self._symbol_count[symbol] += 1
      self._rows[symbol].append(csv_line)
      if self._first_log_timestamp is None:
        self._first_log_timestamp = timestamp
      self._last_log_timestamp = timestamp

  def export_log(self):
    if self._exchange == 'Bitflyer':
      if len(self._balances) > 0:
        ts, bals = zip(*self._balances)
        print(pandas.DatetimeIndex([ts[-1]])[0])
        print("bitflyer est pnl: %s" % (bals[-1] - bals[0]))
        plt.plot(pandas.DatetimeIndex(ts), bals, lw=0.5)
        plt.title("jpy snapshot")
        plt.savefig("oe_stat/%s_bitflyer_jpy.png" % FLAGS.trading_date)
        plt.close()
    for symbol in self._rows.keys():
      self._stat_df[symbol] = pandas.DataFrame(self._rows[symbol], columns=self._header)
      self._stat_oe[symbol] = get_stat_oe(self._stat_df[symbol])
      self._duration_tracker.report(symbol, need_plot=True)
      os.makedirs("oe_stat", exist_ok=True)
      self._stat_df[symbol].to_csv("oe_stat/%s_%s.csv" % (symbol, FLAGS.trading_date))
    print(
        pandas.DatetimeIndex([self._last_log_timestamp])[0],
        "\n",
        pandas.DataFrame(self._stat_oe).to_string())

  def print_futures_volume(self, file):
    assert self._exchange in ['Bitmex', 'Bitflyer']
    volume = {}
    for symbol in self._rows.keys():
      self._stat_df[symbol] = pandas.DataFrame(self._rows[symbol], columns=self._header)
      self._stat_oe[symbol] = get_stat_oe(self._stat_df[symbol], need_plot=False)
      volume[symbol] = {}
      try:
        if self._exchange == 'Bitmex':
          volume[symbol]['volume'] = self._stat_oe[symbol]['fill_qty']
        else:
          volume[symbol]['volume'] = self._stat_oe[symbol]['fill_pq']
      except KeyError:
        volume[symbol]['volume'] = 0

    formats = {}
    formats['volume'] = '{:.0f}'.format
    total_volume = 0
    for key in volume.keys():
      total_volume += volume[key]['volume']

    transposed = pandas.DataFrame(volume).T
    for key, value in formats.items():
      transposed[key] = transposed[key].apply(value)
    print(transposed.to_string(), file=file)

    volume_str = 'totol volume: '
    if self._exchange == 'Bitmex':
      unit = 'USD'
    else:
      unit = 'JPY'
    volume_str += '%.0f %s' % (total_volume, unit)
    print(volume_str, file=file)
    print('first_log_time: %s' % datetime.datetime.fromtimestamp(self._first_log_timestamp / 1e9),
          file=file)
    print('last_log_time: %s' % datetime.datetime.fromtimestamp(self._last_log_timestamp / 1e9),
          file=file)

  def print_pnl_volume(self, file):
    pnl_volume = {}
    for symbol in self._rows.keys():
      self._stat_df[symbol] = pandas.DataFrame(self._rows[symbol], columns=self._header)
      self._stat_oe[symbol] = get_stat_oe(self._stat_df[symbol], need_plot=False)
      pnl_volume[symbol] = {}
      try:
        pnl_volume[symbol]['fill_pq'] = self._stat_oe[symbol]['fill_pq']
        pnl_volume[symbol]['pnl_fill_est'] = self._stat_oe[symbol]['pnl_fill_est']
      except KeyError:
        pnl_volume[symbol]['fill_pq'] = 0
        pnl_volume[symbol]['pnl_fill_est'] = 0

    formats = {}
    total_volume = {}
    total_pnl = {}

    quote_unit = 'BTC'
    for key in pnl_volume.keys():
      unit = key.split('-')[1]
      if unit.upper() == 'BTC':
        format = '{:.3f} %s' % unit
      else:
        quote_unit = unit
        format = '{:.0f} %s' % unit
      # formats[key] = format.format
      total_volume.setdefault(unit, 0)
      total_volume[unit] += pnl_volume[key]['fill_pq']
      total_pnl.setdefault(unit, 0)
      total_pnl[unit] += pnl_volume[key]['pnl_fill_est']

    formats['agg_ratio_cnt'] = '{:.2f}'.format
    formats['agg_ratio_qty'] = '{:.2f}'.format
    formats['fill_qty_ratio'] = '{:.2f}'.format
    formats['full_fill_ratio'] = '{:.2f}'.format
    formats['pnl_fill_ev_bp'] = '{:.4f}'.format
    if quote_unit == 'BTC':
      formats['fill_pq'] = '{:.3f}'.format
      formats['pnl_fill_est'] = '{:.3f}'.format
    else:
      formats['fill_pq'] = '{:.0f}'.format
      formats['pnl_fill_est'] = '{:.0f}'.format
    transposed = pandas.DataFrame(self._stat_oe).T
    for key, value in formats.items():
      transposed[key] = transposed[key].apply(value)
    print(transposed.to_string(), file=file)
    volume_str = 'totol volume: '
    for unit, value in total_volume.items():
      if unit.upper() == 'BTC':
        volume_str += '%.3f %s    ' % (value, unit)
      else:
        volume_str += '%.0f %s    ' % (value, unit)
    print(volume_str, file=file)
    pnl_str = 'totol pnl: '
    for unit, value in total_pnl.items():
      if unit.upper() == 'BTC':
        pnl_str += '%.3f %s    ' % (value, unit)
      else:
        pnl_str += '%.0f %s    ' % (value, unit)
    print(pnl_str, file=file)
    print('first_log_time: %s' % datetime.datetime.fromtimestamp(self._first_log_timestamp / 1e9),
          file=file)
    print('last_log_time: %s' % datetime.datetime.fromtimestamp(self._last_log_timestamp / 1e9),
          file=file)

  def export_log_feed(self):
    for symbol in self._arraydict.keys():
      sdf = pandas.DataFrame(self._stat_df[symbol])
      ndf = pandas.DataFrame(self._arraydict[symbol])
      tdf = pandas.concat([sdf, ndf], axis=1)
      tdf.to_csv("oe_stat/%s_%s_align.csv" % (symbol, FLAGS.trading_date))

  @property
  def products(self):
    if len(self._products) == 0:
      for symbol in self._symbol_count.keys():
        okex_product = OkexProduct.FromStr(symbol)
        huobi_product = HuobiProduct.FromProduct(okex_product)
        binance_product = BinanceProduct.FromProduct(okex_product)

        try:
          feed.OkexFlowBookBuilder.find_feed_group(okex_product)
        except Exception:
          okex_product = None

        try:
          feed.HuobiFlowBookBuilder.find_feed_group(huobi_product)
        except Exception:
          huobi_product = None

        try:
          feed.BinanceFlowBookBuilder.find_feed_group(binance_product)
        except Exception:
          binance_product = None

        products = []

        if okex_product is not None:
          products.append(okex_product)

        if huobi_product is not None:
          products.append(huobi_product)

        if binance_product is not None:
          products.append(binance_product)

        self._symbol_map[symbol] = products
        for product in products:
          self._product_map[product] = symbol
        self._products.extend(products)
    return self._products


def launch(exchange, trading_date, root_dir, machine):
  start_time = trading_date  # UTC 0 hour
  hours = 24

  products = [OkexProduct.FromStr('LTC-USDT')]
  """
  products = [
      OkexProduct.FromStr('BCH-USDT'),
      OkexProduct.FromStr('BTC-USDT'),
      OkexProduct.FromStr('EOS-USDT'),
      OkexProduct.FromStr('LTC-USDT'),
  ]
  """
  strategy = PtaStrategy(products, FLAGS.market_type, exchange)

  run_from_archive(products,
                   strategy.on_book_reset,
                   start_time,
                   start_time + datetime.timedelta(hours=hours),
                   machine=['feed-01.ap-northeast-1.aws'])
  strategy.calculate_period_volatility()

  return 0


def main(_):
  pydevd.settrace('10.81.1.111', port=50000, stdoutToServer=True, stderrToServer=True)
  # import warnings; warnings.filterwarnings("error")
  exchange = FLAGS.exchange
  trading_date = FLAGS.trading_date
  root_dir = os.path.expanduser(FLAGS.root_dir)
  machine = FLAGS.machine
  assert FLAGS.market_type in ['Spot', 'Futures', 'Margin']
  assert trading_date, '--trading_date must be specified.'
  assert machine, '--machine must be specified.'
  assert exchange, '--exchange must be specified.'
  print('Running for %s %s %s ...' % (trading_date, machine, exchange))
  sys.stdout.flush()
  trading_date = datetime.datetime.strptime(trading_date, '%Y%m%d')
  launch(exchange, trading_date, root_dir, machine)


if __name__ == '__main__':
  flags.DEFINE_string('exchange', 'Okex', 'Exchange name.')

  flags.DEFINE_string('trading_date', None, 'Trading date in form of %Y%m%d.')

  flags.DEFINE_string('root_dir', '~/data/og_proto_log', 'root_dir.')

  flags.DEFINE_string('machine', 'strategy-04.aliyun-cn-hongkong', 'Instance machine name.')

  flags.DEFINE_string('market_type', 'Spot', 'Spot, Futures, Margin')

  flags.DEFINE_bool('align_with_feed', False, 'align with feed')

  logging.basicConfig(level='DEBUG', format='%(levelname)8s %(asctime)s %(name)s] %(message)s')
  app.run(main)
